import requests
import pandas as pd
import time
import os, sys
import shutil
import json
import numpy as np
from utitls.UaUtils import get_user_agent
import shutil


from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


# root_path = os.path.dirname(os.path.abspath(__file__))
# sys.path.append(root_path)


class RuidGet(object):
    '''
    配置一个唯一的ruid
    props： None
    return：
    '''
    @classmethod
    def get_str_ruid(cls):
        '''
        获取16进制字符串唯一id
        :return:
        '''
        base_time = round(time.mktime(time.strptime('1970-01-02 00:00:00', '%Y-%m-%d %H:%M:%S'))*10**3)

        ruid = round(time.time()*10**3) - base_time
        time.sleep(0.001)
        return str(hex(ruid)).replace('0x', '')

    @classmethod
    def get_int_ruid(cls):
        '''
        获取10进制整数唯一id
        :return:
        '''
        base_time = round(time.mktime(time.strptime('1970-01-02 00:00:00', '%Y-%m-%d %H:%M:%S')) * 10 ** 3)

        ruid = round(time.time() * 10 ** 3) - base_time
        time.sleep(0.001)
        return str(ruid)

def is_nan(value):
    '''
    判断是否为nan
    :return:
    '''
    if not isinstance(value, str) and np.isnan(value):
        return True
    else:
        return False


if __name__ == '__main__':

    # 使用说明------------------------------------------------
    print('===============================================================')
    print('PS：')
    print('1.文件夹下只放一个excel')
    print('2.excel中需要包含一列“封面url”，下载时会读取这一列的链接')
    print('3.下载前会对excel进行一次备份，保存在bak文件夹下')
    print('4.下载的图片保存在img文件夹下')
    print('5.下载图片时会在excel中新建一列“封面本地保存路径”，将图片路径保存在其中')
    print('===============================================================')
    time.sleep(1)
    # --------------------------------------------------------

    # # 输入操作路径 ----------------------------------------------------------------
    operate_dir = input('请输入采集好的excel目录（旺柴）:') # D:\PycharmProjects\spiders\图片下载工具\excel
    operate_dir = os.path.abspath(operate_dir)
    # operate_dir = os.path.abspath(r'C:\Users\cxstar46\Desktop\采集矫正补充审核好家伙')
    # -----------------------------------------------------------------------------

    # 采集间隔 -----------------------------
    sleep_time = input('请指定采集间隔时间（单位：秒）:')
    sleep_time = int(sleep_time) if sleep_time else None
    # ------------------------------------

    # 是否使用代理 -----------------------------------------------------------------------------------------------------
    proxy_code = input('是否使用代理（如需使用代理，请确保已经打开了小猫咪clash）: \n 1.是 \n 2.否（也可以直接回车，默认不使用代理） \n')
    if proxy_code and proxy_code == '1':
        print('配置代理中...')
        os.environ.update(HTTP_PROXY='http://127.0.0.1:7890', HTTPS_PROXY='http://127.0.0.1:7890')
        print('配置完成！')
    else:
        print('不使用代理噢')
    # ----------------------------------------------------------------------------------------------------------------


    # 读取配置文件 --------------------------------------
    img_url_col = '封面url'
    img_local_path_col = '封面本地保存路径'
    # excel_dir = os.path.abspath('./excel/')
    # img_dir = os.path.abspath('./img/')
    # -------------------------------------------------

    # 创建用来保存封面的img文件夹 --------------------------
    print('创建img文件夹...')
    img_dir_name = 'img'
    img_dir = os.path.join(operate_dir, img_dir_name)
    file_list = os.listdir(operate_dir)
    if img_dir_name not in file_list:
        os.makedirs(img_dir)
    # --------------------------------------------------

    # 检测excel数量，只能放一个，当只有一个excel时，提取它的路径excel_path -------
    print('检查路径下的文件格式...')
    excel_name = None
    excel_count = 0
    file_list = os.listdir(operate_dir)
    for file in file_list:
        if file.endswith('.xlsx') or file.endswith('.xlx'):
            excel_count += 1
            excel_name = file
    if excel_count == 0:
        input('文件夹下没有excel，按任意键退出...')
        raise Exception(0)
    if excel_count > 1:
        input('只能放一个excel，按任意键退出...')
        raise Exception(0)

    # print(excel_name)
    # raise Exception(1212)
    # ----------------------------------------------------------------------

    # 读取excel ----------------------------------------
    excel_path = os.path.join(operate_dir, excel_name)
    # print(excel_path)
    try:
        df = pd.read_excel(excel_path)
    except Exception as e:
        print(e)
        print('文件不存在或已损坏...')
        input('按任意键退出...')
        raise Exception('123')
    # -------------------------------------------------

    # 备份原始excel表 --------------------------------------------------------------
    print('备份excel...')
    bak_dir = 'bak'
    file_list = os.listdir(operate_dir)
    if bak_dir not in file_list:
        os.makedirs(os.path.join(operate_dir, bak_dir))
    bak_excel_path = os.path.join(os.path.join(operate_dir, bak_dir), excel_name)
    shutil.copyfile(excel_path, bak_excel_path)
    # -----------------------------------------------------------------------------

    # # 清空 img文件夹 和 excel表中的 封面保存路径 列 --
    # for file_name in os.listdir(img_dir):
    #     os.remove(img_dir+file_name)
    # df = pd.read_excel(excel_path)
    # df.loc[:, img_local_path_col] = np.nan
    # # print(df.loc[:, '封面本地保存路径'])
    # df.to_excel(excel_path, index=0)
    # # ------------------------------------------

    # 加入 封面保存路径 列 -------------
    if not img_local_path_col in df.columns:
        df.loc[:, img_local_path_col] = np.nan
    # -------------------------------

    # df = df.where(df.notnull(), None)

    for idx in range(df.shape[0]):
        print(f'当前进度： {idx+1} / {df.shape[0]}')


        # 阶段性保存 -----------------------------------------------
        if idx % 200 == 0 and idx != 0:
            print('----- 阶段性保存,此时关闭程序会导致excel损坏 -----')
            df.to_excel(excel_path, index=False)
        # --------------------------------------------------------

        try:
            img_url = df.loc[idx, img_url_col]
        except KeyError as e:
            print(e)
            print('没有找到名称为“封面url”的列')
            input('按任意键继续...')
            raise Exception(123)

        # print(img_url)
        if is_nan(img_url):
            print('没有封面url： {}'.format(idx+1))
            continue

        # 断点续采 ---------------------------------
        img_name = df.loc[idx, img_local_path_col]
        # print(img_name, type(img_name))
        if not is_nan(img_name):
            print('已采过封面： {}'.format(idx+1))
            continue
        # ----------------------------------------


        # 如果指定了 sleep_time ，就歇 sleep_time 秒，防止被封 ----
        if sleep_time:
            time.sleep(sleep_time)
            print(f'休息： {sleep_time} 秒')
        # ---------------------------------------------------


        try:
            headers = {'User-Agent': get_user_agent()}
            res = requests.get(url=img_url, headers=headers, verify=False, timeout=10)
        except Exception as e:
            err_msg = '请求报错 或 超时： {}'.format(e)
            print(err_msg)
            df.loc[idx, img_local_path_col] = err_msg
            continue

        if not res.status_code == 200:
            print('采集失败，ip可能被封了 或者 目标网站服务器故障，code: {}, res: {}'.format(res.status_code, idx))
            continue

        print('采集成功')

        if img_url.endswith(('png', 'PNG')):
            img_name = RuidGet.get_str_ruid() + '.png'
        elif img_url.endswith(('jpg', 'JPG', 'jpeg', 'JPEG')):
            img_name = RuidGet.get_str_ruid() + '.jpg'
        elif img_url.endswith(('gif', 'GIF')):
            img_name = RuidGet.get_str_ruid() + '.gif'
        elif img_url.endswith(('MP4', 'mp4')):
            img_name = RuidGet.get_str_ruid() + '.mp4'
        elif img_url.endswith(('WEBP', 'webp')):
            img_name = RuidGet.get_str_ruid() + '.webp'
        else:
            img_name = RuidGet.get_str_ruid() + '.jpg'
        img_path = os.path.join(img_dir, img_name)

        with open(img_path, mode='wb') as f:
            f.write(res.content)

        df.loc[idx, img_local_path_col] = img_name

    print('正在保存...')
    df.to_excel(excel_path, index=False)
    print('保存完成')
    input('按任意键继续...')